{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Matrix Factorization.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "NbtjMIwt3IK6",
        "colab_type": "code",
        "outputId": "037e6d43-9c33-4b4f-8421-0d7cd5a0a64b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 123
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fs5Apm8a3OGD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from scipy.sparse.linalg import svds\n",
        "from sklearn.decomposition import NMF\n",
        "from sklearn.decomposition import TruncatedSVD"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "anJ4pxDH3Opl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ratings_list = [i.strip().split(\"::\") for i in open('/content/gdrive/My Drive/RLProject/Data/ml-1m/ratings.dat', 'r').readlines()]\n",
        "users_list = [i.strip().split(\"::\") for i in open('/content/gdrive/My Drive/RLProject/Data/ml-1m/users.dat', 'r').readlines()]\n",
        "movies_list = [i.strip().split(\"::\") for i in open('/content/gdrive/My Drive/RLProject/Data/ml-1m/movies.dat',encoding=\"latin-1\").readlines()]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pvd-dmxp7wec",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ratings = np.array(ratings_list)\n",
        "users = np.array(users_list)\n",
        "movies = np.array(movies_list)\n",
        "ratings_df = pd.DataFrame(ratings_list, columns = ['UserID', 'MovieID', 'Rating', 'Timestamp'], dtype = int)\n",
        "movies_df = pd.DataFrame(movies_list, columns = ['MovieID', 'Title', 'Genres'])\n",
        "movies_df['MovieID'] = movies_df['MovieID'].apply(pd.to_numeric)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "otqhJdhZhMoC",
        "colab_type": "code",
        "outputId": "2034d13c-648e-4133-94b5-3ab26a92d711",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(ratings.shape)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1000209, 4)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jfaAzczGSCf7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ratings_df['UserID'] = ratings_df['UserID'].astype(int)\n",
        "ratings_df['MovieID'] = ratings_df['MovieID'].astype(int)\n",
        "ratings_df['Rating'] = ratings_df['Rating'].astype(int)\n",
        "ratings_df['Timestamp'] = ratings_df['Timestamp'].astype(int)\n",
        "ratings_df['MovieID'] = ratings_df['MovieID'].apply(pd.to_numeric)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AcAi56yZ3OsT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "R_df = ratings_df.pivot(index = 'UserID', columns ='MovieID', values = 'Rating').fillna(0)\n",
        "R_df.head()\n",
        "R = R_df.values\n",
        "user_ratings_mean = np.mean(R, axis = 1)\n",
        "R_demeaned = R - user_ratings_mean.reshape(-1, 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "28M3KabH3OvW",
        "colab_type": "code",
        "outputId": "4c8621b3-fa09-4ac0-80b6-d3eb2d80672e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 251
        }
      },
      "source": [
        "U, sigma, Vt = svds(R_demeaned, k = 50)\n",
        "sigma = np.diag(sigma)\n",
        "all_user_predicted_ratings = np.dot(np.dot(U, sigma), Vt) + user_ratings_mean.reshape(-1, 1)\n",
        "preds_df = pd.DataFrame(all_user_predicted_ratings, columns = R_df.columns)\n",
        "preds_df.head()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>MovieID</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>40</th>\n",
              "      <th>...</th>\n",
              "      <th>3913</th>\n",
              "      <th>3914</th>\n",
              "      <th>3915</th>\n",
              "      <th>3916</th>\n",
              "      <th>3917</th>\n",
              "      <th>3918</th>\n",
              "      <th>3919</th>\n",
              "      <th>3920</th>\n",
              "      <th>3921</th>\n",
              "      <th>3922</th>\n",
              "      <th>3923</th>\n",
              "      <th>3924</th>\n",
              "      <th>3925</th>\n",
              "      <th>3926</th>\n",
              "      <th>3927</th>\n",
              "      <th>3928</th>\n",
              "      <th>3929</th>\n",
              "      <th>3930</th>\n",
              "      <th>3931</th>\n",
              "      <th>3932</th>\n",
              "      <th>3933</th>\n",
              "      <th>3934</th>\n",
              "      <th>3935</th>\n",
              "      <th>3936</th>\n",
              "      <th>3937</th>\n",
              "      <th>3938</th>\n",
              "      <th>3939</th>\n",
              "      <th>3940</th>\n",
              "      <th>3941</th>\n",
              "      <th>3942</th>\n",
              "      <th>3943</th>\n",
              "      <th>3944</th>\n",
              "      <th>3945</th>\n",
              "      <th>3946</th>\n",
              "      <th>3947</th>\n",
              "      <th>3948</th>\n",
              "      <th>3949</th>\n",
              "      <th>3950</th>\n",
              "      <th>3951</th>\n",
              "      <th>3952</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4.288861</td>\n",
              "      <td>0.143055</td>\n",
              "      <td>-0.195080</td>\n",
              "      <td>-0.018843</td>\n",
              "      <td>0.012232</td>\n",
              "      <td>-0.176604</td>\n",
              "      <td>-0.074120</td>\n",
              "      <td>0.141358</td>\n",
              "      <td>-0.059553</td>\n",
              "      <td>-0.195950</td>\n",
              "      <td>0.512867</td>\n",
              "      <td>-0.089172</td>\n",
              "      <td>0.310181</td>\n",
              "      <td>-0.002005</td>\n",
              "      <td>-0.052401</td>\n",
              "      <td>-0.189827</td>\n",
              "      <td>0.238360</td>\n",
              "      <td>0.006466</td>\n",
              "      <td>-0.099315</td>\n",
              "      <td>-0.069682</td>\n",
              "      <td>-0.321492</td>\n",
              "      <td>0.111577</td>\n",
              "      <td>0.034795</td>\n",
              "      <td>0.320576</td>\n",
              "      <td>-0.118217</td>\n",
              "      <td>-0.012647</td>\n",
              "      <td>0.065573</td>\n",
              "      <td>-0.098318</td>\n",
              "      <td>0.064081</td>\n",
              "      <td>-0.005914</td>\n",
              "      <td>0.091936</td>\n",
              "      <td>0.180563</td>\n",
              "      <td>-0.009566</td>\n",
              "      <td>2.641693</td>\n",
              "      <td>-0.012495</td>\n",
              "      <td>0.765179</td>\n",
              "      <td>0.019784</td>\n",
              "      <td>0.002917</td>\n",
              "      <td>0.053079</td>\n",
              "      <td>0.014856</td>\n",
              "      <td>...</td>\n",
              "      <td>0.018810</td>\n",
              "      <td>-0.018782</td>\n",
              "      <td>0.022249</td>\n",
              "      <td>0.227852</td>\n",
              "      <td>-0.067653</td>\n",
              "      <td>-0.046039</td>\n",
              "      <td>-0.023574</td>\n",
              "      <td>-0.019405</td>\n",
              "      <td>-0.005116</td>\n",
              "      <td>-0.032921</td>\n",
              "      <td>-0.008259</td>\n",
              "      <td>-0.019157</td>\n",
              "      <td>0.007527</td>\n",
              "      <td>-0.008687</td>\n",
              "      <td>-0.025630</td>\n",
              "      <td>-0.013563</td>\n",
              "      <td>0.015240</td>\n",
              "      <td>-0.044665</td>\n",
              "      <td>-0.009568</td>\n",
              "      <td>-0.043549</td>\n",
              "      <td>-0.003131</td>\n",
              "      <td>-0.008221</td>\n",
              "      <td>-0.005948</td>\n",
              "      <td>0.031885</td>\n",
              "      <td>-0.003424</td>\n",
              "      <td>-0.001159</td>\n",
              "      <td>-0.002124</td>\n",
              "      <td>-0.002827</td>\n",
              "      <td>0.010393</td>\n",
              "      <td>-0.001068</td>\n",
              "      <td>0.027807</td>\n",
              "      <td>0.001640</td>\n",
              "      <td>0.026395</td>\n",
              "      <td>-0.022024</td>\n",
              "      <td>-0.085415</td>\n",
              "      <td>0.403529</td>\n",
              "      <td>0.105579</td>\n",
              "      <td>0.031912</td>\n",
              "      <td>0.050450</td>\n",
              "      <td>0.088910</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.744716</td>\n",
              "      <td>0.169659</td>\n",
              "      <td>0.335418</td>\n",
              "      <td>0.000758</td>\n",
              "      <td>0.022475</td>\n",
              "      <td>1.353050</td>\n",
              "      <td>0.051426</td>\n",
              "      <td>0.071258</td>\n",
              "      <td>0.161601</td>\n",
              "      <td>1.567246</td>\n",
              "      <td>0.772656</td>\n",
              "      <td>0.046179</td>\n",
              "      <td>-0.054562</td>\n",
              "      <td>0.042344</td>\n",
              "      <td>0.048390</td>\n",
              "      <td>0.347313</td>\n",
              "      <td>1.074905</td>\n",
              "      <td>-0.099782</td>\n",
              "      <td>0.008163</td>\n",
              "      <td>0.250869</td>\n",
              "      <td>2.186638</td>\n",
              "      <td>0.018789</td>\n",
              "      <td>-0.002199</td>\n",
              "      <td>0.218934</td>\n",
              "      <td>0.824475</td>\n",
              "      <td>0.139274</td>\n",
              "      <td>-0.007135</td>\n",
              "      <td>0.053071</td>\n",
              "      <td>-0.156952</td>\n",
              "      <td>0.044739</td>\n",
              "      <td>-0.002960</td>\n",
              "      <td>0.453298</td>\n",
              "      <td>-0.007484</td>\n",
              "      <td>0.920325</td>\n",
              "      <td>0.016566</td>\n",
              "      <td>1.335129</td>\n",
              "      <td>-0.015066</td>\n",
              "      <td>-0.045602</td>\n",
              "      <td>0.034649</td>\n",
              "      <td>0.122010</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.042363</td>\n",
              "      <td>-0.137822</td>\n",
              "      <td>-0.112071</td>\n",
              "      <td>0.380783</td>\n",
              "      <td>-0.036273</td>\n",
              "      <td>-0.016174</td>\n",
              "      <td>0.002920</td>\n",
              "      <td>-0.148021</td>\n",
              "      <td>-0.017614</td>\n",
              "      <td>-0.033474</td>\n",
              "      <td>0.086133</td>\n",
              "      <td>0.008153</td>\n",
              "      <td>-0.126819</td>\n",
              "      <td>0.109208</td>\n",
              "      <td>0.001798</td>\n",
              "      <td>0.151866</td>\n",
              "      <td>0.014118</td>\n",
              "      <td>0.032897</td>\n",
              "      <td>0.005764</td>\n",
              "      <td>0.042259</td>\n",
              "      <td>0.022404</td>\n",
              "      <td>0.003260</td>\n",
              "      <td>0.010556</td>\n",
              "      <td>0.137181</td>\n",
              "      <td>-0.042184</td>\n",
              "      <td>0.006759</td>\n",
              "      <td>-0.005789</td>\n",
              "      <td>0.000340</td>\n",
              "      <td>0.002024</td>\n",
              "      <td>0.016013</td>\n",
              "      <td>-0.056502</td>\n",
              "      <td>-0.013733</td>\n",
              "      <td>-0.010580</td>\n",
              "      <td>0.062576</td>\n",
              "      <td>-0.016248</td>\n",
              "      <td>0.155790</td>\n",
              "      <td>-0.418737</td>\n",
              "      <td>-0.101102</td>\n",
              "      <td>-0.054098</td>\n",
              "      <td>-0.140188</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.818824</td>\n",
              "      <td>0.456136</td>\n",
              "      <td>0.090978</td>\n",
              "      <td>-0.043037</td>\n",
              "      <td>-0.025694</td>\n",
              "      <td>-0.158617</td>\n",
              "      <td>-0.131778</td>\n",
              "      <td>0.098977</td>\n",
              "      <td>0.030551</td>\n",
              "      <td>0.735470</td>\n",
              "      <td>-0.023476</td>\n",
              "      <td>0.034796</td>\n",
              "      <td>0.065942</td>\n",
              "      <td>0.008661</td>\n",
              "      <td>0.110348</td>\n",
              "      <td>-0.002952</td>\n",
              "      <td>-0.122061</td>\n",
              "      <td>0.063974</td>\n",
              "      <td>0.061033</td>\n",
              "      <td>0.081799</td>\n",
              "      <td>0.329471</td>\n",
              "      <td>0.149579</td>\n",
              "      <td>0.095352</td>\n",
              "      <td>-0.161493</td>\n",
              "      <td>0.022545</td>\n",
              "      <td>-0.009284</td>\n",
              "      <td>-0.002677</td>\n",
              "      <td>-0.142710</td>\n",
              "      <td>0.012345</td>\n",
              "      <td>-0.085331</td>\n",
              "      <td>0.076139</td>\n",
              "      <td>-0.355795</td>\n",
              "      <td>-0.008579</td>\n",
              "      <td>1.046871</td>\n",
              "      <td>-0.088946</td>\n",
              "      <td>0.383583</td>\n",
              "      <td>-0.018144</td>\n",
              "      <td>-0.038618</td>\n",
              "      <td>0.113984</td>\n",
              "      <td>0.006942</td>\n",
              "      <td>...</td>\n",
              "      <td>0.007233</td>\n",
              "      <td>-0.047221</td>\n",
              "      <td>0.066474</td>\n",
              "      <td>-0.179455</td>\n",
              "      <td>0.097428</td>\n",
              "      <td>0.034113</td>\n",
              "      <td>0.008098</td>\n",
              "      <td>-0.024784</td>\n",
              "      <td>-0.012749</td>\n",
              "      <td>-0.007394</td>\n",
              "      <td>-0.017220</td>\n",
              "      <td>0.004719</td>\n",
              "      <td>0.113348</td>\n",
              "      <td>-0.074943</td>\n",
              "      <td>-0.145795</td>\n",
              "      <td>0.128619</td>\n",
              "      <td>0.112567</td>\n",
              "      <td>0.045500</td>\n",
              "      <td>-0.018027</td>\n",
              "      <td>-0.058946</td>\n",
              "      <td>-0.002770</td>\n",
              "      <td>-0.035276</td>\n",
              "      <td>-0.008085</td>\n",
              "      <td>0.132182</td>\n",
              "      <td>-0.017005</td>\n",
              "      <td>0.014383</td>\n",
              "      <td>0.006598</td>\n",
              "      <td>-0.006217</td>\n",
              "      <td>-0.000342</td>\n",
              "      <td>0.000518</td>\n",
              "      <td>0.040481</td>\n",
              "      <td>-0.005301</td>\n",
              "      <td>0.012832</td>\n",
              "      <td>0.029349</td>\n",
              "      <td>0.020866</td>\n",
              "      <td>0.121532</td>\n",
              "      <td>0.076205</td>\n",
              "      <td>0.012345</td>\n",
              "      <td>0.015148</td>\n",
              "      <td>-0.109956</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.408057</td>\n",
              "      <td>-0.072960</td>\n",
              "      <td>0.039642</td>\n",
              "      <td>0.089363</td>\n",
              "      <td>0.041950</td>\n",
              "      <td>0.237753</td>\n",
              "      <td>-0.049426</td>\n",
              "      <td>0.009467</td>\n",
              "      <td>0.045469</td>\n",
              "      <td>-0.111370</td>\n",
              "      <td>-0.375831</td>\n",
              "      <td>0.068658</td>\n",
              "      <td>0.011199</td>\n",
              "      <td>0.069699</td>\n",
              "      <td>-0.037529</td>\n",
              "      <td>-0.238788</td>\n",
              "      <td>0.060607</td>\n",
              "      <td>-0.043418</td>\n",
              "      <td>0.053152</td>\n",
              "      <td>0.078237</td>\n",
              "      <td>0.357185</td>\n",
              "      <td>-0.096005</td>\n",
              "      <td>-0.028243</td>\n",
              "      <td>-0.067169</td>\n",
              "      <td>0.246164</td>\n",
              "      <td>-0.020379</td>\n",
              "      <td>0.034461</td>\n",
              "      <td>-0.022225</td>\n",
              "      <td>-0.012327</td>\n",
              "      <td>0.009182</td>\n",
              "      <td>0.014730</td>\n",
              "      <td>0.215893</td>\n",
              "      <td>-0.019687</td>\n",
              "      <td>-0.293933</td>\n",
              "      <td>-0.011511</td>\n",
              "      <td>0.145326</td>\n",
              "      <td>-0.029213</td>\n",
              "      <td>0.030029</td>\n",
              "      <td>-0.045409</td>\n",
              "      <td>-0.030684</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.015077</td>\n",
              "      <td>-0.030208</td>\n",
              "      <td>0.028357</td>\n",
              "      <td>-0.072643</td>\n",
              "      <td>-0.135727</td>\n",
              "      <td>-0.053318</td>\n",
              "      <td>-0.012962</td>\n",
              "      <td>-0.054465</td>\n",
              "      <td>0.005870</td>\n",
              "      <td>-0.018048</td>\n",
              "      <td>-0.006836</td>\n",
              "      <td>-0.008222</td>\n",
              "      <td>-0.027214</td>\n",
              "      <td>-0.071677</td>\n",
              "      <td>-0.094072</td>\n",
              "      <td>-0.010745</td>\n",
              "      <td>-0.103191</td>\n",
              "      <td>-0.031297</td>\n",
              "      <td>-0.023920</td>\n",
              "      <td>-0.015053</td>\n",
              "      <td>-0.017914</td>\n",
              "      <td>-0.029561</td>\n",
              "      <td>-0.024299</td>\n",
              "      <td>-0.057678</td>\n",
              "      <td>-0.111450</td>\n",
              "      <td>-0.015473</td>\n",
              "      <td>-0.007123</td>\n",
              "      <td>-0.007416</td>\n",
              "      <td>-0.011508</td>\n",
              "      <td>-0.010038</td>\n",
              "      <td>0.008571</td>\n",
              "      <td>-0.005425</td>\n",
              "      <td>-0.008500</td>\n",
              "      <td>-0.003417</td>\n",
              "      <td>-0.083982</td>\n",
              "      <td>0.094512</td>\n",
              "      <td>0.057557</td>\n",
              "      <td>-0.026050</td>\n",
              "      <td>0.014841</td>\n",
              "      <td>-0.034224</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.574272</td>\n",
              "      <td>0.021239</td>\n",
              "      <td>-0.051300</td>\n",
              "      <td>0.246884</td>\n",
              "      <td>-0.032406</td>\n",
              "      <td>1.552281</td>\n",
              "      <td>-0.199630</td>\n",
              "      <td>-0.014920</td>\n",
              "      <td>-0.060498</td>\n",
              "      <td>0.450512</td>\n",
              "      <td>-0.251178</td>\n",
              "      <td>0.012337</td>\n",
              "      <td>-0.084051</td>\n",
              "      <td>0.258937</td>\n",
              "      <td>0.016570</td>\n",
              "      <td>0.980536</td>\n",
              "      <td>1.267869</td>\n",
              "      <td>0.275619</td>\n",
              "      <td>-0.008139</td>\n",
              "      <td>-0.038832</td>\n",
              "      <td>1.849627</td>\n",
              "      <td>0.107649</td>\n",
              "      <td>-0.168424</td>\n",
              "      <td>0.386541</td>\n",
              "      <td>1.790343</td>\n",
              "      <td>0.192379</td>\n",
              "      <td>-0.054356</td>\n",
              "      <td>0.267566</td>\n",
              "      <td>1.027817</td>\n",
              "      <td>0.374665</td>\n",
              "      <td>-0.010445</td>\n",
              "      <td>1.947980</td>\n",
              "      <td>0.017468</td>\n",
              "      <td>2.784035</td>\n",
              "      <td>0.274397</td>\n",
              "      <td>1.422393</td>\n",
              "      <td>0.040553</td>\n",
              "      <td>0.022926</td>\n",
              "      <td>1.345800</td>\n",
              "      <td>0.104507</td>\n",
              "      <td>...</td>\n",
              "      <td>0.075475</td>\n",
              "      <td>0.330767</td>\n",
              "      <td>0.150470</td>\n",
              "      <td>-0.261636</td>\n",
              "      <td>0.085163</td>\n",
              "      <td>-0.014229</td>\n",
              "      <td>-0.029247</td>\n",
              "      <td>0.124172</td>\n",
              "      <td>0.092875</td>\n",
              "      <td>0.061895</td>\n",
              "      <td>0.034757</td>\n",
              "      <td>0.054386</td>\n",
              "      <td>0.047055</td>\n",
              "      <td>0.048403</td>\n",
              "      <td>0.082926</td>\n",
              "      <td>0.129035</td>\n",
              "      <td>-0.174646</td>\n",
              "      <td>0.102727</td>\n",
              "      <td>0.024732</td>\n",
              "      <td>0.047280</td>\n",
              "      <td>0.017818</td>\n",
              "      <td>0.041451</td>\n",
              "      <td>0.041595</td>\n",
              "      <td>-0.007138</td>\n",
              "      <td>-0.080448</td>\n",
              "      <td>0.018639</td>\n",
              "      <td>0.034068</td>\n",
              "      <td>0.026941</td>\n",
              "      <td>0.035905</td>\n",
              "      <td>0.024459</td>\n",
              "      <td>0.110151</td>\n",
              "      <td>0.046010</td>\n",
              "      <td>0.006934</td>\n",
              "      <td>-0.015940</td>\n",
              "      <td>-0.050080</td>\n",
              "      <td>-0.052539</td>\n",
              "      <td>0.507189</td>\n",
              "      <td>0.033830</td>\n",
              "      <td>0.125706</td>\n",
              "      <td>0.199244</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 3706 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "MovieID      1         2         3     ...      3950      3951      3952\n",
              "0        4.288861  0.143055 -0.195080  ...  0.031912  0.050450  0.088910\n",
              "1        0.744716  0.169659  0.335418  ... -0.101102 -0.054098 -0.140188\n",
              "2        1.818824  0.456136  0.090978  ...  0.012345  0.015148 -0.109956\n",
              "3        0.408057 -0.072960  0.039642  ... -0.026050  0.014841 -0.034224\n",
              "4        1.574272  0.021239 -0.051300  ...  0.033830  0.125706  0.199244\n",
              "\n",
              "[5 rows x 3706 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VhcXAh773zKa",
        "colab_type": "code",
        "outputId": "cef0bf46-0733-4b0d-e5aa-7edd8d4def97",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 415
        }
      },
      "source": [
        "def recommend_movies(predictions_df, userID, movies_df, original_ratings_df, num_recommendations=5):\n",
        "    user_row_number = userID - 1 # UserID starts at 1, not 0\n",
        "    sorted_user_predictions = preds_df.iloc[user_row_number].sort_values(ascending=False) # UserID starts at 1\n",
        "    user_data = original_ratings_df[original_ratings_df.UserID == (userID)]\n",
        "    user_full = (user_data.merge(movies_df, how = 'left', left_on = 'MovieID', right_on = 'MovieID').\n",
        "                     sort_values(['Rating'], ascending=False)\n",
        "                 )\n",
        "    print(user_full.head())\n",
        "    print('User {0} has already rated {1} movies.'.format(userID, user_full.shape[0]))\n",
        "    print('Recommending highest {0} predicted ratings movies not already rated.'.format(num_recommendations))\n",
        "    # Recommend the highest predicted rating movies that the user hasn't seen yet.\n",
        "    recommendations = movies_df[~movies_df[\"MovieID\"].isin(user_full['MovieID'])]\n",
        "    recommendations = recommendations.merge(pd.DataFrame(sorted_user_predictions).reset_index(), how = 'left',\n",
        "               left_on = 'MovieID',\n",
        "               right_on = 'MovieID')\n",
        "    recommendations = recommendations.rename(columns = {user_row_number: 'Predictions'})\n",
        "    recommendations = recommendations.sort_values(['Predictions'], ascending = False)\n",
        "    recommendations = recommendations.iloc[:num_recommendations, :-1]\n",
        "\n",
        "    return user_full, recommendations\n",
        "already_rated, predictions = recommend_movies(preds_df, 837, movies_df, ratings_df, 10)\n",
        "print(predictions)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    UserID  MovieID  ...                           Title              Genres\n",
            "36     837      858  ...           Godfather, The (1972)  Action|Crime|Drama\n",
            "35     837     1387  ...                     Jaws (1975)       Action|Horror\n",
            "65     837     2028  ...      Saving Private Ryan (1998)    Action|Drama|War\n",
            "63     837     1221  ...  Godfather: Part II, The (1974)  Action|Crime|Drama\n",
            "11     837      913  ...      Maltese Falcon, The (1941)   Film-Noir|Mystery\n",
            "\n",
            "[5 rows x 6 columns]\n",
            "User 837 has already rated 69 movies.\n",
            "Recommending highest 10 predicted ratings movies not already rated.\n",
            "      MovieID  ...                       Genres\n",
            "516       527  ...                    Drama|War\n",
            "1848     1953  ...  Action|Crime|Drama|Thriller\n",
            "596       608  ...         Crime|Drama|Thriller\n",
            "1235     1284  ...            Film-Noir|Mystery\n",
            "2085     2194  ...           Action|Crime|Drama\n",
            "1188     1230  ...               Comedy|Romance\n",
            "1198     1242  ...             Action|Drama|War\n",
            "897       922  ...                    Film-Noir\n",
            "1849     1954  ...                 Action|Drama\n",
            "581       593  ...               Drama|Thriller\n",
            "\n",
            "[10 rows x 3 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oKBXKXgDMtJU",
        "colab_type": "code",
        "outputId": "c3641387-c343-4541-d41b-6468d09b5ad8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 390
        }
      },
      "source": [
        "\n",
        "already_rated.head(10)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>UserID</th>\n",
              "      <th>MovieID</th>\n",
              "      <th>Rating</th>\n",
              "      <th>Timestamp</th>\n",
              "      <th>Title</th>\n",
              "      <th>Genres</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>837</td>\n",
              "      <td>858</td>\n",
              "      <td>5</td>\n",
              "      <td>975360036</td>\n",
              "      <td>Godfather, The (1972)</td>\n",
              "      <td>Action|Crime|Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>837</td>\n",
              "      <td>1387</td>\n",
              "      <td>5</td>\n",
              "      <td>975360036</td>\n",
              "      <td>Jaws (1975)</td>\n",
              "      <td>Action|Horror</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65</th>\n",
              "      <td>837</td>\n",
              "      <td>2028</td>\n",
              "      <td>5</td>\n",
              "      <td>975360089</td>\n",
              "      <td>Saving Private Ryan (1998)</td>\n",
              "      <td>Action|Drama|War</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63</th>\n",
              "      <td>837</td>\n",
              "      <td>1221</td>\n",
              "      <td>5</td>\n",
              "      <td>975360036</td>\n",
              "      <td>Godfather: Part II, The (1974)</td>\n",
              "      <td>Action|Crime|Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>837</td>\n",
              "      <td>913</td>\n",
              "      <td>5</td>\n",
              "      <td>975359921</td>\n",
              "      <td>Maltese Falcon, The (1941)</td>\n",
              "      <td>Film-Noir|Mystery</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>837</td>\n",
              "      <td>3417</td>\n",
              "      <td>5</td>\n",
              "      <td>975360893</td>\n",
              "      <td>Crimson Pirate, The (1952)</td>\n",
              "      <td>Adventure|Comedy|Sci-Fi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>837</td>\n",
              "      <td>2186</td>\n",
              "      <td>4</td>\n",
              "      <td>975359955</td>\n",
              "      <td>Strangers on a Train (1951)</td>\n",
              "      <td>Film-Noir|Thriller</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>55</th>\n",
              "      <td>837</td>\n",
              "      <td>2791</td>\n",
              "      <td>4</td>\n",
              "      <td>975360893</td>\n",
              "      <td>Airplane! (1980)</td>\n",
              "      <td>Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>837</td>\n",
              "      <td>1188</td>\n",
              "      <td>4</td>\n",
              "      <td>975360920</td>\n",
              "      <td>Strictly Ballroom (1992)</td>\n",
              "      <td>Comedy|Romance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>837</td>\n",
              "      <td>1304</td>\n",
              "      <td>4</td>\n",
              "      <td>975360058</td>\n",
              "      <td>Butch Cassidy and the Sundance Kid (1969)</td>\n",
              "      <td>Action|Comedy|Western</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    UserID  ...                   Genres\n",
              "36     837  ...       Action|Crime|Drama\n",
              "35     837  ...            Action|Horror\n",
              "65     837  ...         Action|Drama|War\n",
              "63     837  ...       Action|Crime|Drama\n",
              "11     837  ...        Film-Noir|Mystery\n",
              "20     837  ...  Adventure|Comedy|Sci-Fi\n",
              "34     837  ...       Film-Noir|Thriller\n",
              "55     837  ...                   Comedy\n",
              "31     837  ...           Comedy|Romance\n",
              "28     837  ...    Action|Comedy|Western\n",
              "\n",
              "[10 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nJ5vTj4ASlkF",
        "colab_type": "code",
        "outputId": "516e413f-3b7e-4742-cd0a-fb5ecc186cce",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 355
        }
      },
      "source": [
        "predictions"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>MovieID</th>\n",
              "      <th>Title</th>\n",
              "      <th>Genres</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>516</th>\n",
              "      <td>527</td>\n",
              "      <td>Schindler's List (1993)</td>\n",
              "      <td>Drama|War</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1848</th>\n",
              "      <td>1953</td>\n",
              "      <td>French Connection, The (1971)</td>\n",
              "      <td>Action|Crime|Drama|Thriller</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>596</th>\n",
              "      <td>608</td>\n",
              "      <td>Fargo (1996)</td>\n",
              "      <td>Crime|Drama|Thriller</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1235</th>\n",
              "      <td>1284</td>\n",
              "      <td>Big Sleep, The (1946)</td>\n",
              "      <td>Film-Noir|Mystery</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2085</th>\n",
              "      <td>2194</td>\n",
              "      <td>Untouchables, The (1987)</td>\n",
              "      <td>Action|Crime|Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1188</th>\n",
              "      <td>1230</td>\n",
              "      <td>Annie Hall (1977)</td>\n",
              "      <td>Comedy|Romance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1198</th>\n",
              "      <td>1242</td>\n",
              "      <td>Glory (1989)</td>\n",
              "      <td>Action|Drama|War</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>897</th>\n",
              "      <td>922</td>\n",
              "      <td>Sunset Blvd. (a.k.a. Sunset Boulevard) (1950)</td>\n",
              "      <td>Film-Noir</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1849</th>\n",
              "      <td>1954</td>\n",
              "      <td>Rocky (1976)</td>\n",
              "      <td>Action|Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>581</th>\n",
              "      <td>593</td>\n",
              "      <td>Silence of the Lambs, The (1991)</td>\n",
              "      <td>Drama|Thriller</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      MovieID  ...                       Genres\n",
              "516       527  ...                    Drama|War\n",
              "1848     1953  ...  Action|Crime|Drama|Thriller\n",
              "596       608  ...         Crime|Drama|Thriller\n",
              "1235     1284  ...            Film-Noir|Mystery\n",
              "2085     2194  ...           Action|Crime|Drama\n",
              "1188     1230  ...               Comedy|Romance\n",
              "1198     1242  ...             Action|Drama|War\n",
              "897       922  ...                    Film-Noir\n",
              "1849     1954  ...                 Action|Drama\n",
              "581       593  ...               Drama|Thriller\n",
              "\n",
              "[10 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Gj50Jesr9iy",
        "colab_type": "text"
      },
      "source": [
        "## **Model Evaluation**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nOdWrBulSqzz",
        "colab_type": "code",
        "outputId": "24ea1464-bd35-429c-b333-e13dd4ec5591",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        }
      },
      "source": [
        "!pip install surprise"
      ],
      "execution_count": 104,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: surprise in /usr/local/lib/python3.6/dist-packages (0.1)\n",
            "Requirement already satisfied: scikit-surprise in /usr/local/lib/python3.6/dist-packages (from surprise) (1.1.0)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.12.0)\n",
            "Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.2 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.18.3)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (0.14.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "olx8xaaObWO8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from surprise import SVD\n",
        "from surprise import SVDpp\n",
        "from surprise import Dataset\n",
        "from surprise.model_selection import cross_validate\n",
        "from surprise.model_selection import KFold\n",
        "from collections import defaultdict"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9JBjLcsUvZv-",
        "colab_type": "code",
        "outputId": "ba7f7170-f99d-41ff-c98e-d61ca3cac50b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 398
        }
      },
      "source": [
        "\n",
        "# Load the dataset (download it if needed)\n",
        "data = Dataset.load_builtin('ml-1m')\n",
        "kf = KFold(n_splits = 5)\n",
        "# Use the famous SVD algorithm\n",
        "algo = SVD()\n",
        "\n",
        "# Run 5-fold cross-validation and then print results\n",
        "cross_validate(algo, data, measures=['RMSE', 'MAE'], cv=5, verbose=True)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Dataset ml-1m could not be found. Do you want to download it? [Y/n] y\n",
            "Trying to download dataset from http://files.grouplens.org/datasets/movielens/ml-1m.zip...\n",
            "Done! Dataset ml-1m has been saved to /root/.surprise_data/ml-1m\n",
            "Evaluating RMSE, MAE of algorithm SVD on 5 split(s).\n",
            "\n",
            "                  Fold 1  Fold 2  Fold 3  Fold 4  Fold 5  Mean    Std     \n",
            "RMSE (testset)    0.8726  0.8756  0.8743  0.8741  0.8699  0.8733  0.0019  \n",
            "MAE (testset)     0.6849  0.6873  0.6864  0.6852  0.6826  0.6853  0.0016  \n",
            "Fit time          51.27   52.01   51.95   52.33   52.12   51.94   0.36    \n",
            "Test time         1.70    2.19    2.23    1.73    2.14    2.00    0.24    \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'fit_time': (51.27277684211731,\n",
              "  52.01305365562439,\n",
              "  51.95322942733765,\n",
              "  52.333860635757446,\n",
              "  52.1150643825531),\n",
              " 'test_mae': array([0.68491612, 0.6872737 , 0.68636962, 0.6851536 , 0.68263057]),\n",
              " 'test_rmse': array([0.87259056, 0.87559995, 0.8742955 , 0.87411295, 0.86993232]),\n",
              " 'test_time': (1.6960456371307373,\n",
              "  2.1940250396728516,\n",
              "  2.2317709922790527,\n",
              "  1.728822946548462,\n",
              "  2.141263484954834)}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5hYXjWufvZzP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def precision_recall_at_k(predictions, k=10 ,threshold=4):\n",
        "    user_est_true = defaultdict(list)\n",
        "    for uid, _, true_r, est, _ in predictions:\n",
        "        user_est_true[uid].append((est, true_r))\n",
        "\n",
        "    precisions = dict()\n",
        "    recalls = dict()\n",
        "    for uid, user_ratings in user_est_true.items():\n",
        "        user_ratings.sort(key=lambda x: x[0], reverse=True)\n",
        "        n_rel = sum((true_r >= threshold) for (_, true_r) in user_ratings)\n",
        "        n_rec_k = sum((est >= threshold) for (est, _) in user_ratings[:k])\n",
        "        n_rel_and_rec_k = sum(((true_r >= threshold) and (est >= threshold))\n",
        "                              for (est, true_r) in user_ratings[:k])\n",
        "        precisions[uid] = n_rel_and_rec_k / n_rec_k if n_rec_k != 0 else 1\n",
        "        recalls[uid] = n_rel_and_rec_k / n_rel if n_rel != 0 else 1\n",
        "    return precisions, recalls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lDUimGGNvZ2c",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_P_R_F(data, algo):\n",
        "    for trainset, testset in kf.split(data):\n",
        "        algo.fit(trainset)\n",
        "        predictions = algo.test(testset)\n",
        "        precisions, recalls = precision_recall_at_k(predictions, k = 5, threshold = 4)\n",
        "        P = sum(prec for prec in precisions.values()) / len(precisions)\n",
        "        R = sum(rec for rec in recalls.values()) / len(recalls)\n",
        "        F1 = 2 * P * R / (P + R)\n",
        "        print('precision : ', P)\n",
        "        print('recall : ', R)\n",
        "        print('F1 : '  , F1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dEoie1FBzQr5",
        "colab_type": "code",
        "outputId": "cdf1c41e-03cc-4659-ee0a-31ac4f43588f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 277
        }
      },
      "source": [
        "algo = SVD(n_factors = 10, n_epochs = 1, biased = False, lr_all = 0.009, reg_all = 0.2)\n",
        "get_P_R_F(data, algo)"
      ],
      "execution_count": 144,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "precision :  0.721877763041557\n",
            "recall :  0.18532016607288798\n",
            "F1 :  0.294926834900898\n",
            "precision :  0.7209969621651487\n",
            "recall :  0.18405891853340198\n",
            "F1 :  0.293254646375114\n",
            "precision :  0.724630120348902\n",
            "recall :  0.18464249020120188\n",
            "F1 :  0.2942957004172187\n",
            "precision :  0.7283603931963776\n",
            "recall :  0.1888855204012758\n",
            "F1 :  0.29997785734246\n",
            "precision :  0.7242750621375321\n",
            "recall :  0.19092730290056836\n",
            "F1 :  0.3021930219035299\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YW8SPRtmTEI1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}
